// NewsInfoGetter.java

import javafx.util.Pair;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

import java.sql.Timestamp;
import java.util.ArrayList;

/**
 * 新闻基本信息爬取类
 */
public class NewsInfoGetter {
    private static final String URL = "http://news.baidu.com/";

    /**
     * 新闻基本信息
     */
    static class News {
        private String title;   // 新闻标题
        private String category;    // 新闻所属类别
        private Timestamp collect_time; // 新闻采集时间
        private String href;    // 新闻链接

        public News(String title, String category, Timestamp collect_time, String href) {
            this.title = title;
            this.category = category;
            this.collect_time = collect_time;
            this.href = href;
        }

        public String getTitle() {
            return title;
        }

        public void setTitle(String title) {
            this.title = title;
        }

        public String getCategory() {
            return category;
        }

        public void setCategory(String category) {
            this.category = category;
        }

        public Timestamp getCollect_time() {
            return collect_time;
        }

        public void setCollect_time(Timestamp collect_time) {
            this.collect_time = collect_time;
        }

        public String getHref() {
            return href;
        }

        public void setHref(String href) {
            this.href = href;
        }
    }

    /**
     * 爬取网页，获取新闻标题及所属类别
     *
     * @return 包含新闻信息（{@link News}）的 {@code ArrayList}
     * @throws Exception
     */
    public static ArrayList<News> getNews() throws Exception {
        // categoryList：第一个 String 是新闻类别（汉字），第二个 String 是类别对应的网页链接
        ArrayList<Pair<String, String>> categoryList = new ArrayList<>();
        HttpUtils httpUtils = HttpUtils.getInstance();
        httpUtils.setCharset("UTF-8");
        httpUtils.setTimeout(30000);
        // 获取百度新闻首页 html 源码
        Document document = httpUtils.executeGetAsDocument(URL);
        // 解析 html，获取网页头部导航栏中各栏目名及网址，添加到 categoryList 中
        for (Element categoryLink : document.getElementById("channel-all").child(0).child(0).children()) {
            categoryList.add(
                    new Pair<>(categoryLink.text(),
                            URL.endsWith("/")
                                    ? URL.substring(0, URL.length() - 1) + categoryLink.select("a").attr("href")
                                    : URL + categoryLink.select("a").attr("href")
                    ));
        }
        // returnList：暂存返回值
        ArrayList<News> returnList = new ArrayList<>();
        for (Pair<String, String> category : categoryList) {
            // 获取每个分类对应网址的 html 源码
            Document documentByType = httpUtils.executeGetAsDocument(category.getValue());
            // 部分新闻标题被包围在 <ul class="ulist xxx" xxx> </ul> 块中
            for (Element element : documentByType.select("ul")) {
                if (element.attr("class").contains("ulist")) {
                    // 格式形如 <a href="xxx" xxx>新闻标题</a>
                    for (Element singleNews : element.select("a")) {
                        returnList.add(new News(
                                singleNews.text(),
                                category.getKey(),
                                new Timestamp(System.currentTimeMillis()),
                                singleNews.select("a").attr("href").replace(" ", "")
                        ));
                    }
                }
            }
            // 部分新闻标题被包围在 <div class="tlc"> </div> 或 <div class="auto-text-ul"> </div> 块中
            for (Element element : documentByType.select("div")) {
                if (element.attr("class").contains("tlc") ||
                        element.attr("class").contains("auto-text-ul")) {
                    // 格式形如 <a href="xxx" xxx>新闻标题</a>
                    for (Element singleNews : element.select("a")) {
                        returnList.add(new News(
                                singleNews.text(),
                                category.getKey(),
                                new Timestamp(System.currentTimeMillis()),
                                singleNews.select("a").attr("href")
                        ));
                    }
                }
            }
        }
        return returnList;
    }
}
